<!DOCTYPE html>
<html lang="en-us">
  <head>

    <meta http-equiv="content-type" content="text/html; charset=utf-8">
    
<meta charset="UTF-8">
<title>Time-Based Data | Elasticsearch: The Definitive Guide [2.x] | Elastic</title>
<link rel="home" href="index.html" title="Elasticsearch: The Definitive Guide [2.x]">
<link rel="up" href="scale.html" title="Designing for Scale">
<link rel="prev" href="multiple-indices.html" title="Multiple Indices">
<link rel="next" href="index-templates.html" title="Index Templates">
<meta name="DC.type" content="Learn/Docs/Legacy/Elasticsearch/Definitive Guide/2.x">
<meta name="DC.subject" content="Elasticsearch">
<meta name="DC.identifier" content="2.x">
<meta name="robots" content="noindex,nofollow">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <script src="https://cdn.optimizely.com/js/18132920325.js"></script>
    <link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
    <link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
    <link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
    <link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
    <link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
    <link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
    <link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
    <link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
    <link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
    <link rel="icon" type="image/png" href="/favicon-32x32.png" sizes="32x32">
    <link rel="icon" type="image/png" href="/android-chrome-192x192.png" sizes="192x192">
    <link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96">
    <link rel="icon" type="image/png" href="/favicon-16x16.png" sizes="16x16">
    <link rel="manifest" href="/manifest.json">
    <meta name="apple-mobile-web-app-title" content="Elastic">
    <meta name="application-name" content="Elastic">
    <meta name="msapplication-TileColor" content="#ffffff">
    <meta name="msapplication-TileImage" content="/mstile-144x144.png">
    <meta name="theme-color" content="#ffffff">
    <meta name="naver-site-verification" content="936882c1853b701b3cef3721758d80535413dbfd">
    <meta name="yandex-verification" content="d8a47e95d0972434">
    <meta name="localized" content="true">
    <meta name="st:robots" content="follow,index">
    <meta property="og:image" content="https://www.elastic.co/static/images/elastic-logo-200.png">
    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
    <link rel="icon" href="/favicon.ico" type="image/x-icon">
    <link rel="apple-touch-icon-precomposed" sizes="64x64" href="/favicon_64x64_16bit.png">
    <link rel="apple-touch-icon-precomposed" sizes="32x32" href="/favicon_32x32.png">
    <link rel="apple-touch-icon-precomposed" sizes="16x16" href="/favicon_16x16.png">
    <!-- Give IE8 a fighting chance -->
    <!--[if lt IE 9]>
    <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
    <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
    <![endif]-->
    <link rel="stylesheet" type="text/css" href="/guide/static/styles.css">
  </head>

  <!--© 2015-2021 Elasticsearch B.V. Copying, publishing and/or distributing without written permission is strictly prohibited.-->

  <body>
    <!-- Google Tag Manager -->
    <script>dataLayer = [];</script><noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-58RLH5" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
    <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= '//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-58RLH5');</script>
    <!-- End Google Tag Manager -->

    <!-- Global site tag (gtag.js) - Google Analytics -->
    <script async src="https://www.googletagmanager.com/gtag/js?id=UA-12395217-16"></script>
    <script>
      window.dataLayer = window.dataLayer || [];
      function gtag(){dataLayer.push(arguments);}
      gtag('js', new Date());
      gtag('config', 'UA-12395217-16');
    </script>

    <!--BEGIN QUALTRICS WEBSITE FEEDBACK SNIPPET-->
    <script type="text/javascript">
      (function(){var g=function(e,h,f,g){
      this.get=function(a){for(var a=a+"=",c=document.cookie.split(";"),b=0,e=c.length;b<e;b++){for(var d=c[b];" "==d.charAt(0);)d=d.substring(1,d.length);if(0==d.indexOf(a))return d.substring(a.length,d.length)}return null};
      this.set=function(a,c){var b="",b=new Date;b.setTime(b.getTime()+6048E5);b="; expires="+b.toGMTString();document.cookie=a+"="+c+b+"; path=/; "};
      this.check=function(){var a=this.get(f);if(a)a=a.split(":");else if(100!=e)"v"==h&&(e=Math.random()>=e/100?0:100),a=[h,e,0],this.set(f,a.join(":"));else return!0;var c=a[1];if(100==c)return!0;switch(a[0]){case "v":return!1;case "r":return c=a[2]%Math.floor(100/c),a[2]++,this.set(f,a.join(":")),!c}return!0};
      this.go=function(){if(this.check()){var a=document.createElement("script");a.type="text/javascript";a.src=g;document.body&&document.body.appendChild(a)}};
      this.start=function(){var a=this;window.addEventListener?window.addEventListener("load",function(){a.go()},!1):window.attachEvent&&window.attachEvent("onload",function(){a.go()})}};
      try{(new g(100,"r","QSI_S_ZN_emkP0oSe9Qrn7kF","https://znemkp0ose9qrn7kf-elastic.siteintercept.qualtrics.com/WRSiteInterceptEngine/?Q_ZID=ZN_emkP0oSe9Qrn7kF")).start()}catch(i){}})();
    </script><div id="ZN_emkP0oSe9Qrn7kF"><!--DO NOT REMOVE-CONTENTS PLACED HERE--></div>
    <!--END WEBSITE FEEDBACK SNIPPET-->

    <div id="elastic-nav" style="display:none;"></div>
    <script src="https://www.elastic.co/elastic-nav.js"></script>

    <!-- Subnav -->
    <div>
      <div>
        <div class="tertiary-nav d-none d-md-block">
          <div class="container">
            <div class="p-t-b-15 d-flex justify-content-between nav-container">
              <div class="breadcrum-wrapper"><span><a href="/guide/" style="font-size: 14px; font-weight: 600; color: #000;">Docs</a></span></div>
            </div>
          </div>
        </div>
      </div>
    </div>

    <div class="main-container">
      <section id="content">
        <div class="content-wrapper">

          <section id="guide" lang="en">
            <div class="container">
              <div class="row">
                <div class="col-xs-12 col-sm-8 col-md-8 guide-section">
                  <!-- start body -->
                  <div class="page_header">
<p>
  <strong>WARNING</strong>: The 2.x versions of Elasticsearch have passed their
  <a href="https://www.elastic.co/support/eol">EOL dates</a>. If you are running
  a 2.x version, we strongly advise you to upgrade.
</p>
<p>
  This documentation is no longer maintained and may be removed. For the latest
  information, see the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html">current
  Elasticsearch documentation</a>.
</p>
</div>
<div id="content">
<div class="breadcrumbs">
<span class="breadcrumb-link"><a href="index.html">Elasticsearch: The Definitive Guide [2.x]</a></span>
»
<span class="breadcrumb-link"><a href="modeling-your-data.html">Modeling Your Data</a></span>
»
<span class="breadcrumb-link"><a href="scale.html">Designing for Scale</a></span>
»
<span class="breadcrumb-node">Time-Based Data</span>
</div>
<div class="navheader">
<span class="prev">
<a href="multiple-indices.html">« Multiple Indices</a>
</span>
<span class="next">
<a href="index-templates.html">Index Templates »</a>
</span>
</div>
<div class="section">
<div class="titlepage"><div><div>
<h2 class="title">
<a id="time-based"></a>Time-Based Data<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/410_Scaling/45_Index_per_timeframe.asciidoc">edit</a>
</h2>
</div></div></div>
<p>One of the most common use cases for Elasticsearch is for logging, so common
in fact that Elasticsearch provides an integrated logging platform called the
<em>ELK stack</em>—Elasticsearch, Logstash, and Kibana—​to make the process easy.</p>
<p><a href="/guide/en/logstash/current/index.html" class="ulink" target="_top">Logstash</a> collects, parses, and
enriches logs before indexing them into Elasticsearch.  Elasticsearch acts as
a centralized logging server, and
<a href="/guide/en/kibana/current/index.html" class="ulink" target="_top">Kibana</a> is a graphic frontend
that makes it easy to query and visualize what is happening across your
network in near real-time.</p>
<p>Most traditional use cases for search engines involve a relatively static
collection of documents that grows slowly. Searches look for the most relevant
documents, regardless of when they were created.</p>
<p>Logging—​and other time-based data streams such as social-network activity—​are very different in nature.  The number of documents in the index grows
rapidly, often accelerating with time.  Documents are almost never updated,
and searches mostly target the most recent documents.  As documents age, they
lose value.</p>
<p>We need to adapt our index design to function with the flow of time-based
data.</p>
<div class="section">
<div class="titlepage"><div><div>
<h3 class="title">
<a id="index-per-timeframe"></a>Index per Time Frame<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/410_Scaling/45_Index_per_timeframe.asciidoc">edit</a>
</h3>
</div></div></div>
<p>If we were to have one big index for documents of this type, we would soon run
out of space. Logging events just keep on coming, without pause or
interruption. We could delete the old events with a <a class="xref" href="scroll.html" title="Scroll"><code class="literal">scroll</code></a>
query and bulk delete,  but this approach is <em>very inefficient</em>.  When you delete a
document, it is only <em>marked</em> as deleted (see <a class="xref" href="dynamic-indices.html#deletes-and-updates" title="Deletes and Updates">Deletes and Updates</a>). It won’t
be physically deleted until the segment containing it is merged away.</p>
<p>Instead, use an <em>index per time frame</em>. You could start out with an index per
year (<code class="literal">logs_2014</code>) or per month (<code class="literal">logs_2014-10</code>).  Perhaps, when your
website gets really busy, you need to switch to an index per day
(<code class="literal">logs_2014-10-24</code>).  Purging old data is easy: just delete old indices.</p>
<p>This approach has the advantage of allowing you to scale as and when you need
to.  You don’t have to make any difficult decisions up front.  Every day is a
new opportunity to change your indexing time frames to suit the current demand.
Apply the same logic to how big you make each index.  Perhaps all you need is
one primary shard per week initially.  Later, maybe you need five primary shards
per day.  It doesn’t matter—​you can adjust to new circumstances at any
time.</p>
<p>Aliases can help make switching indices more transparent.  For indexing,
you can point <code class="literal">logs_current</code> to the index currently accepting new log events,
and for searching, update <code class="literal">last_3_months</code> to point to all indices for the
previous three months:</p>
<div class="pre_wrapper lang-json">
<pre class="programlisting prettyprint lang-json">POST /_aliases
{
  "actions": [
    { "add":    { "alias": "logs_current",  "index": "logs_2014-10" }}, <a id="CO291-1"></a><i class="conum" data-value="1"></i>
    { "remove": { "alias": "logs_current",  "index": "logs_2014-09" }}, <a id="CO291-2"></a><i class="conum" data-value="1"></i>
    { "add":    { "alias": "last_3_months", "index": "logs_2014-10" }}, <a id="CO291-3"></a><i class="conum" data-value="2"></i>
    { "remove": { "alias": "last_3_months", "index": "logs_2014-07" }}  <a id="CO291-4"></a><i class="conum" data-value="2"></i>
  ]
}</pre>
</div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO291-1"><i class="conum" data-value="1"></i></a><a href="#CO291-2"></a></p>
</td>
<td align="left" valign="top">
<p>Switch <code class="literal">logs_current</code> from September to October.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO291-3"><i class="conum" data-value="2"></i></a><a href="#CO291-4"></a></p>
</td>
<td align="left" valign="top">
<p>Add October to <code class="literal">last_3_months</code> and remove July.</p>
</td>
</tr>
</table>
</div>
</div>

</div>
<div class="navfooter">
<span class="prev">
<a href="multiple-indices.html">« Multiple Indices</a>
</span>
<span class="next">
<a href="index-templates.html">Index Templates »</a>
</span>
</div>
</div>

                  <!-- end body -->
                </div>
                <div class="col-xs-12 col-sm-4 col-md-4" id="right_col">
                  <div id="rtpcontainer" style="display: block;">
                    <div class="mktg-promo">
                      <h3>Most Popular</h3>
                      <ul class="icons">
                        <li class="icon-elasticsearch-white"><a href="https://www.elastic.co/webinars/getting-started-elasticsearch?baymax=default&amp;elektra=docs&amp;storm=top-video">Get Started with Elasticsearch: Video</a></li>
                        <li class="icon-kibana-white"><a href="https://www.elastic.co/webinars/getting-started-kibana?baymax=default&amp;elektra=docs&amp;storm=top-video">Intro to Kibana: Video</a></li>
                        <li class="icon-logstash-white"><a href="https://www.elastic.co/webinars/introduction-elk-stack?baymax=default&amp;elektra=docs&amp;storm=top-video">ELK for Logs &amp; Metrics: Video</a></li>
                      </ul>
                    </div>
                  </div>
                </div>
              </div>
            </div>
          </section>

        </div>


<div id="elastic-footer"></div>
<script src="https://www.elastic.co/elastic-footer.js"></script>
<!-- Footer Section end-->

      </section>
    </div>

<script src="/guide/static/jquery.js"></script>
<script type="text/javascript" src="/guide/static/docs.js"></script>
<script type="text/javascript">
  window.initial_state = {}</script>
  </body>
</html>
